# -*- coding: utf-8 -*-
#
#
# libshorttext does not support chinese for new :(
# failed experiment
#
#
import os

def Convert2SVMData(_dir,label,_output_file, validate_rate=0.2):

    all_context=[]
    for filename in os.listdir(_dir):
        one_f = open(_dir + "/" + filename,"r")
        context = one_f.read()
        all_context.append(context.replace("\n","").replace("\r","").replace(" ","").replace("\t",""))
        one_f.close()

    splited_line_index = int(len(all_context) * validate_rate)

    f_output = open(_output_file+".test", "a")
    for i in range(0,splited_line_index):
        f_output.write(str(label)+" "+ all_context[i] + "\n")
    f_output.close()

    f_output = open(_output_file+".train", "a")
    for i in range(splited_line_index,len(all_context)):
        f_output.write(str(label)+"\t"+ all_context[i] + "\n")
    f_output.close()
    pass

import jieba
from libshorttext.classifier import *
from libshorttext.converter import *
from libshorttext.analyzer import *
from Data import DataProcess

text_converter = Text2svmConverter()
text_converter.text_prep.tokenizer = jieba.cut

def preprocess():
    #step1
    Convert2SVMData("dataset/raw/Auto", 0,"dataset/svm/corpus.1.txt",0)
    Convert2SVMData("dataset/raw/Culture", 1, "dataset/svm/corpus.1.txt",0)
    Convert2SVMData("dataset/raw/Economy", 2, "dataset/svm/corpus.1.txt",0)
    Convert2SVMData("dataset/raw/Medicine", 3, "dataset/svm/corpus.1.txt",0)
    Convert2SVMData("dataset/raw/Military", 4, "dataset/svm/corpus.1.txt",0)
    Convert2SVMData("dataset/raw/Sports", 5, "dataset/svm/corpus.1.txt",0)
    #step2
    convert_text("dataset/svm/corpus.1.txt.train", text_converter, "dataset/svm/corpus.1.txt.input")
    DataProcess.shuffle_line_in_file("dataset/svm/corpus.1.txt.input", "dataset/svm/corpus.1.txt.input.shuffle")

#preprocess()

def training():

    #m = train_converted_text("dataset/svm/corpus.1.txt.input", text_converter,0 ,"","-v 10")
    m = train_converted_text("dataset/svm/corpus.1.txt.input", text_converter)
    #Cross Validation Accuracy = 98.0917%
    m.save("dataset/svm/model")
    pass

#training()

if __name__ == "__main__":

    textModel = TextModel("dataset/svm/model")
    #textModel.text_converter = text_converter

    analyzer = Analyzer(textModel)
    print analyzer.analyze_single("足球")

    print predict_single_text("足球", textModel)
    print predict_single_text("足球草地绿色",textModel)
    print predict_single_text("银行资金投资",textModel)
    pass